package com.eric.utils.page;

import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.fnlp.nlp.cn.ner.TimeNormalizer;
import org.fnlp.nlp.cn.ner.TimeUnit;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.eric.Config;
import com.eric.utils.UrlUtils;

/**
 * 解析html页面，获取一些文本信息，以方便进行IR分析
 * 
 * @author Eric
 *
 */
public class IR {

	public static String getImgSrc(Element element, URL URL) {
		String imgSrc = null;
		if (element.hasAttr("onclick")) {
			for (Attribute a : element.attributes()) {

				if (!a.getKey().equals("src") && a.getValue().matches(".*\\.jpg")) {
					imgSrc = element.attr(a.getKey());
					break;
				} else {
					imgSrc = element.attr("src");
				}
			}
		} else {
			imgSrc = element.attr("src");
		}

		if (!imgSrc.startsWith("http")) {
			if (!imgSrc.startsWith("/")) {
				imgSrc = URL.getHost() + "/" + imgSrc;
			} else {
				imgSrc = URL.getHost() + imgSrc;
			}
		}
		return imgSrc;
	}

	public static boolean isDetailsPage(Document doc) {
		if (hasH1(doc) && StringUtils.contains(doc.title(), doc.select("h1").text()))
			return true;
		else
			return false;
	}

	public static boolean hasH1(Document doc) {
		Elements h1 = doc.select("h1");
		if (h1 != null) {
			if (StringUtils.isNotBlank(h1.text()))
				return true;
			else
				return false;
		} else
			return false;
	}

	/**
	 * 关键词频率,term frequency 取全部正文，然后算词频
	 * 
	 * @param kw
	 * @return int 词频
	 */
	public static float tf(String kw, String content) {
		Pattern p = Pattern.compile(kw);
		Matcher m = p.matcher(content);
		int c = 0;
		while (!m.hitEnd() && m.find()) {
			c++;
		}
		return (float) c / content.length();
	}

	public static List<String> anchors(Document doc) {
		Elements linkElements = doc.select("a");
		return anchors(linkElements);
	}

	public static List<String> anchors(Elements linkElements) {
		List<String> list = new ArrayList<>();
		Set<String> set = new HashSet<>();
		for (Element e : linkElements) {
			set.add(e.text());
		}
		list.addAll(set);
		return list;
	}

	public static List<Map<String, String>> linkElements(Elements elements, String host) {
		List<Map<String, String>> list = new ArrayList<>();
		Elements e = elements.select("a");
		for (Element element : e) {
			list.add(linkElement(element, host));
		}
		return list;
	}

	/**
	 * 如果传入的不是Link Element则返回空的map
	 * 
	 * @param element
	 * @return
	 */
	public static Map<String, String> linkElement(Element element, String host) {
		Map<String, String> map = new HashMap<>();
		if (element.select("a").size() > 0) {
			for (Attribute a : element.attributes()) {
				String key = a.getKey();
				String value = a.getValue();
				if (key.equals("href")) {
					value = UrlUtils.handleUrl(value);
				}
				if (value.matches("^/.*"))
					value = host + value;
				map.put(key, value);
			}

			if (map.get("href") != null) {
				if (map.get("href").startsWith("http") && UrlUtils.isExternal(map.get("href"), host)) {
					map.put("external", "yes");
				} else {
					map.put("external", "no");
				}
			}

			map.put("anchor", element.text());

			if (element.select("img").size() > 0) {
				map.put("linkType", "图片链接");
			} else if (element.hasAttr("style") && StringUtils.isBlank(element.text())
					&& element.attr("style").toLowerCase().contains("url")) {// 放在style里的背景链接图片
				map.put("linkType", "图片链接");
			} else {
				map.put("linkType", "文字链接");
			}

			if (map.get("rel") != null && map.get("rel").toLowerCase().equals("nofollow")) {
				map.put("followType", "nofollow");
				map.remove("rel");
			} else {
				map.put("followType", "follow");
			}
		}

		return map;
	}

	public static List<String> boldText(Document doc) {
		Elements elements = doc.getAllElements();
		return boldText(elements);
	}

	public static List<String> boldText(Elements elements) {
		List<String> list = new ArrayList<>();
		Set<String> set = new HashSet<>();
		for (Element e : elements) {
			if (e.tagName().equals("b"))
				set.add(e.text());
		}
		list.addAll(set);
		return list;
	}

	/**
	 * 获取页面内以列表形式展现的文本
	 * 
	 * @param elements
	 * @return
	 */
	@SuppressWarnings("unchecked")
	public static HashMap<String, ArrayList<String>> listings(Elements elements) {
		HashMap<String, ArrayList<String>> map = new HashMap<String, ArrayList<String>>();
		for (Element e : elements) {
			ArrayList<String> list = new ArrayList<String>();
			HashSet<String> set = new HashSet<String>();

			// 判断是否为列表，依据是：列表的tagName相同
			if (e.children().size() > 1) {
				for (Element e1 : e.children()) {
					set.add(e1.tagName());
				}
			}

			if (set.size() == 1) {
				Elements elements2 = Jsoup.parseBodyFragment(e.html()).getAllElements();// children()方法无法遍历子、孙元素，只能遍历直接子元素
				for (Element e2 : elements2) {
					if (e2.text().equals(e2.ownText()) && StringUtils.isNotBlank(e2.ownText())) {
						list.add(e2.ownText().trim());
					}
				}
				map.put(e.tagName() + e.siblingIndex(), (ArrayList<String>) list.clone());
				list = null;
			}
		}
		return map;
	}

	/**
	 * 获取页面内以列表形式展现的文本 合并列表下，不同element的文本
	 * 
	 * @param elements
	 * @return
	 */
	@SuppressWarnings("unchecked")
	public static HashMap<String, ArrayList<String>> simpleListings(Elements elements) {
		HashMap<String, ArrayList<String>> map = new HashMap<String, ArrayList<String>>();
		for (Element e : elements) {
			ArrayList<String> list = new ArrayList<String>();
			HashSet<String> set = new HashSet<String>();

			// 判断是否为列表，依据是：列表的tagName相同
			if (e.children().size() > 1) {
				for (Element e1 : e.children()) {
					if (e1.text().length() > Config.LISTING_WORDS_LIMIT) {
						set = null;
						break;
					}
					set.add(e1.tagName());
				}
			}

			if (set != null && set.size() == 1) {
				for (Element e2 : e.children()) {
					list.add(e2.text().trim());
				}
				map.put(e.tagName() + e.siblingIndex(), (ArrayList<String>) list.clone());
				list = null;
			}
		}
		return map;
	}

	@SuppressWarnings("unchecked")
	public static HashMap<String, ArrayList<String>> simpleListingsNew(Elements elements) {
		HashMap<String, ArrayList<String>> map = new HashMap<String, ArrayList<String>>();
		for (Element e : elements) {
			ArrayList<String> list = new ArrayList<String>();
			if (e.siblingElements() != null && e.parent() != null && e.select("a").size() < 3
					&& !e.parent().tagName().equals("body")) {// 判断兄弟元素是否存在，且包含的a标签不超过2个
				for (Element es : e.siblingElements()) {// 遍历所有兄弟元素
					if (IR.compareElement(e, es)) {// 判断2个元素是否对等，是则表示为列表
						for (Element a : e.parent().select("a"))// 取a标签（列表必定是链接）
						{
							if (StringUtils.isNotBlank(a.text()))
								list.add(a.text());
						}
						break;
					}
				}
				if (list.size() > 1)
					map.put(e.parent().tagName() + "#" + elementOffset(e.parent()), (ArrayList<String>) list.clone());
			}
		}
		return map;
	}

	public static boolean isUniqChild(Element element) {
		int c = 0;
		if (element.parent().children().size() == 1)
			return true;
		for (Element e : element.parent().children()) {
			if (e.tagName().equals(element.tagName()))
				c++;
		}
		if (c == 1)
			return true;
		else
			return false;
	}

	/**
	 * 根据tagName和attribute判断2个不同的element是否对等，不同于equals函数
	 * 
	 * @param element1
	 * @param element2
	 * @return
	 */
	public static boolean compareElement(Element element1, Element element2) {
		if (!element1.tagName().equals(element2.tagName()))
			return false;
		for (Attribute a : element1.attributes()) {
			if (element2.attr(a.getKey()).contains(a.getValue()) || a.getValue().contains(element2.attr(a.getKey())))
				return true;
			else
				return false;
		}
		return false;
	}

	public static String tag(Element element) {
		StringBuilder sb = new StringBuilder();
		sb.append(element.tagName());
		for (Attribute a : element.attributes()) {
			sb.append(" " + a.getKey() + "=" + "\"" + a.getValue() + "\"");
		}
		return sb.toString();
	}

	public static int elementOffset(Element element) {
		return element.ownerDocument().html().indexOf(tag(element));
	}

	public static String text(Document doc, boolean articleOnly) {
		if (articleOnly) {
			ArticleExtractor ex = new ArticleExtractor(doc);
			return ex.extract();
		} else {
			return doc.text();
		}
	}

	public static boolean isArticlePage(Document doc) {
		boolean t = false;
		ArticleExtractor ex = new ArticleExtractor(doc);
		if (ex.extract().replaceAll(Config.REGEX_SEPARATOR, "").length() > Config.ARTICLE_WORDS_COUNT)
			t = true;
		return t;
	}

	/**
	 * 如果页面没有定义h1，则用这个方法：<br>
	 * 提取页面的meta title，然后看页面中出现的位置就是h1 title
	 * 
	 * @param doc
	 * @return
	 */
	public static Element findH1(Document doc) {
		Element element = null;
		if (h1(doc) == null) {
			String title = findArticleTitle(doc).trim();
			for (Element e : doc.body().getAllElements()) {
				if (e.ownText().contains(title)) {
					element = e;
				} else {

				}
			}
		} else {
			element = h1(doc);
		}

		return element;
	}

	/**
	 * 去掉meta title里的自动后缀
	 * 
	 * @param doc
	 * @return
	 */
	public static String findArticleTitle(Document doc) {
		String title = null;
		String metaTitle = doc.title();
		int end = 0;
		for (int i = 0; i < metaTitle.length(); i++) {
			String titleIn = metaTitle.substring(0, i);
			int count = StringUtils.countMatches(doc.body().html(), titleIn);// 找到能在页面内匹配的最大长度的meta
																				// title
			if (count > 0) {
				end = i;
			}
		}
		title = metaTitle.substring(0, end);
		return title;
	}

	public static Element h1(Document doc) {
		Element h1 = null;
		String title = doc.title();
		for (Element element : doc.select("h1")) {
			if (StringUtils.contains(title, element.text())) {
				h1 = element;
			}
		}
		return h1;
	}

	// /**
	// * 使用Fudan Nlp包提取文章发布时间
	// *
	// * @deprecated
	// * @param doc
	// * @return
	// */
	// public static Date getTime(Document doc) {
	// Element e = IR.findH1(doc);
	// if (e.hasText()) {
	// String text = e.text();
	// while (getTime(text) == null) {
	// e = e.parent();
	// text = e.text();
	// }
	// return getTime(text);
	// } else {
	// return null;
	// }
	//
	// }

	/**
	 * 使用Fudan Nlp 包进行提取发布时间
	 * 
	 * @param doc
	 * @return
	 */
	public static Date getTime(Document doc) {
		String text = doc.body().text();
		if (IR.findH1(doc) != null) {
			String h1 = IR.findH1(doc).text();// 找到h1
			int begin = text.indexOf(h1) + h1.length();
			int end = begin + 150;

			if (end > text.length())// h1位于页面末尾，则一般不是文章页
				return null;

			String text2 = text.substring(begin, end).replaceAll(h1, "");// 获取紧跟h1文本后的若干文本，不包含h1（h1可能包含时间描述，但不是文章发布时间）

			// TimeNormalizer normalizer = new TimeNormalizer("time.m");
			// normalizer.parse(text2);
			// TimeUnit[] unit = normalizer.getTimeUnit();
			//
			// return unit[0].getTime();
			return getTimeByRegex(text2);
		} else {
			return null;
		}
	}

	public static Date getTimeFromUrl(String strUrl, TimeNormalizer normalizer) {
		normalizer.parse(strUrl.replaceAll("/", " "));
		TimeUnit[] unit = normalizer.getTimeUnit();
		return unit[0].getTime();
	}

	public static Date getTimeByRegex(String dateString) {
		String format = extractTimeString(dateString).replaceAll("[./年月日]", "-");
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		try {
			return sdf.parse(format);
		} catch (ParseException e) {
			e.printStackTrace();
		}
		return null;
	}

	public static String extractTimeString(String input) {
		Pattern p = Pattern.compile("[0-9]{4}[年/.-][0-9]{1,2}[月/.-][0-9]{1,2}[日]?");
		Matcher m = p.matcher(input);
		m.find();
		String result = null;
		try {
			result = m.group();
		} catch (IllegalStateException e) {

		}
		return result;
	}

	public static int countSeparators(String text) {
		String[] s = text.split("");
		int count = 0;
		for (int i = 0; i < s.length; i++) {
			if (s[i].matches(Config.REGEX_CN_SEPARATOR)) {
				count++;
			}
		}
		return count;
	}

	/**
	 * 返回element.class#id作为key，文本和node比例（浮点）为值的map<br>
	 * 如果class或者id没有，则不附加到element名称后
	 * 
	 * @param doc
	 * @return
	 */
	public static Map<String, Float> getNodeTextRatio(Document doc) {
		Map<String, Float> map = new HashMap<>();
		ValueComparator bvc = new ValueComparator(map);
		TreeMap<String, Float> sorted_map = new TreeMap<String, Float>(bvc);

		for (Element element : doc.getAllElements()) {
			int nodeCount = getNodeCount(element);

			if (element.hasText() && element.text().length() > Config.Min_PARAGRAPH_WORDS && nodeCount > 2) {// element不是最底部的，且包含文字，且大于指定个字
				if (element.text().contains("。")) {// element要包含句子分隔符

					StringBuilder sb = new StringBuilder();
					sb.append(element.tagName());
					if (StringUtils.isNotBlank(element.className())) {
						sb.append("." + element.className().split(" ")[0]);// TODO
																			// 暂时只取第一个className
					}
					if (StringUtils.isNotBlank(element.id())) {
						sb.append("#" + element.id());
					}
					map.put(sb.toString(), (float) (element.text().length() / nodeCount));
				}
			}

		}
		sorted_map.putAll(map);
		return sorted_map;
	}

	public static int getNodeCount(Element element) {
		int c = 0;
		for (@SuppressWarnings("unused")
		Element e : element.getAllElements()) {
			c++;
		}
		return c;
	}

}

class ValueComparator implements Comparator<String> {

	Map<String, Float> base;

	public ValueComparator(Map<String, Float> base) {
		this.base = base;
	}

	// Note: this comparator imposes orderings that are inconsistent with
	// equals.
	public int compare(String a, String b) {
		if (base.get(a) >= base.get(b)) {
			return -1;
		} else {
			return 1;
		} // returning 0 would merge keys
	}
}
